# coding:utf8
import copy
import re
from .exceptions import ConfigValueMissingException, ConfigValueErrorException
from multiprocessing import Manager
from .utils import select_use_db_conf, replace_str

FIXED_TYPE = "fixed"
DB_TYPE = "database"
ALL_TYPE = [
    FIXED_TYPE,
    DB_TYPE
]
HTTP_URL_RE = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
QUEUE_NUM_DEFAULT = 10000

class UrlManager(object):
    def __init__(self, url_conf, queue_num=QUEUE_NUM_DEFAULT):
        self._conf = url_conf
        self.url_conf = copy.deepcopy(self._conf)
        self.url_queue = Manager().Queue(queue_num)
        self.new_urls = set()
        self.old_urls = set()

    def init_url(self):
        self.url_conf = copy.deepcopy(self._conf)
        self.verify_conf()
        if self.url_conf.get("type") == FIXED_TYPE:
            self.add_fixed_url()
        elif self.url_conf.get("type") == DB_TYPE:
            self.add_db_url()
        else:
            raise Exception('url_conf中的type不合规')

    def add_fixed_url(self):  # 增加固定的url列表
        if isinstance(self.url_conf['value'], str):
            self.url_conf['value'] = [self.url_conf['value']]
        if isinstance(self.url_conf['value'], list) or isinstance(self.url_conf['value'], set):
            url_list = replace_str(self.url_conf['value'], self.url_conf.get('replace'))
            self.add_url_queues(url_list)
        else:
            raise Exception('url_conf中的value不合规')

    def add_db_url(self):  # 从数据库增加urls
        db_info_list = select_use_db_conf(self.url_conf['db_conf'])
        url_list = [replace_str(replace_str(self.url_conf['value'], i), self.url_conf.get('replace')) for i in db_info_list]
        self.add_url_queues(url_list)

    def add_url_queue(self, url):  # 向url队列里增加一个url
        if url is None:
            return
        self.url_queue.put(url)

    def add_url_queues(self, urls):  # 向url队列里批量添加url
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_url_queue(url)

    def q_size(self):  # 返回队列大小
        return self.url_queue.qsize()

    def empty(self):  # 返回队列是否为空
        return self.url_queue.empty()

    def add_new_url(self, url):  # 向管理器中添加一个新的url
        if url is None:
            return
        if url not in self.new_urls and url not in self.old_urls:
            self.new_urls.add(url)

    def add_new_urls(self, urls):  # 向管理器中添加批量的url
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_new_url(url)

    def has_new_url(self):  # 判断管理器中是否有新的待爬取的url
        return len(self.new_urls) != 0

    def get_new_url(self):  # 从url管理器中获取一个新的待爬取的url
        new_url = self.new_urls.pop()
        self.old_urls.add(new_url)
        return new_url

    def verify_conf(self):
        if self.url_conf.get("type") is None:
            raise ConfigValueMissingException("type")
        if self.url_conf.get("type") not in ALL_TYPE:
            raise ConfigValueErrorException("type")
        if self.url_conf.get("type") == FIXED_TYPE:
            if self.url_conf.get('value') is None:
                raise ConfigValueMissingException("value")
            # if not HTTP_URL_RE.match(self.url_conf.get("value")):
            #     raise ConfigValueErrorException("fixed")
        elif self.url_conf.get("type") == DB_TYPE:
            if self.url_conf.get("db_conf") is None:
                raise ConfigValueMissingException("db_conf")
            if not isinstance(self.url_conf['db_conf'], dict):
                raise ConfigValueErrorException("db_conf")
            for conf in ["table"]:
                if self.url_conf["db_conf"].get(conf) is None:
                    raise ConfigValueMissingException(conf)

